package com.blackman.skcrawler.core.crawler.worm;


import com.blackman.skcrawler.common.util.IpUtils;
import com.blackman.skcrawler.core.crawler.api.IDefaultCrawler;
import com.blackman.skcrawler.core.crawler.api.IDefaultCrawlerCallback;
import com.blackman.skcrawler.core.crawler.po.WormGrabedPage;
import com.blackman.skcrawler.core.crawler.po.WormHomePage;
import com.blackman.skcrawler.core.crawler.po.WormPageRegular;
import com.blackman.skcrawler.core.crawler.po.WormPendPage;
import com.blackman.skcrawler.core.crawler.vo.CrawlerParam;
import com.blackman.skcrawler.core.http.api.IDefautHttpProxyData;
import com.blackman.skcrawler.core.zookeeper.DistributedLock;
import com.blackman.skcrawler.core.crawler.api.IDefaultDataCallBack;
import org.apache.commons.lang3.StringUtils;

import java.util.*;

/**
 * 基本策略的实现类
 *
 * @author staywood
 * @date 2018/04/20 15:25
 *
 */
public class DefaultCrawler extends CrawlerParam implements IDefaultCrawler {

    // 中转注入service接口
    private HashMap<String, Object> dataObjMap;

    // 缓存抓取好了的待抓取列表
    protected List<String> newsCachePendPageList = new ArrayList<String>();

    protected IDefaultCrawlerCallback defaultCrawerCallback;

    protected IDefautHttpProxyData defautHttpProxyData;

    protected IDefaultDataCallBack defaultDataCallBack;

    public DefaultCrawler(){

    }

    /**
     * 爬虫基本配置初始化
     */
    public void init() {
        // 起始页初始化
        defaultDataCallBack.upHomePage(getHomePageList());
        // 起始页初始化加入带抓取列表
        addGrabList(getHomePageList(), 1);
        // 将匹配页面默认加入种子页面
        this.getSeedList().addAll(getRegularList());
        // 种子初始化
        defaultDataCallBack.upPageRegular(this.getSeedList(), WormPageRegular.REGULAR_SEED);
        this.setSeedList(defaultDataCallBack.findSeedListByType(WormPageRegular.REGULAR_SEED));
        // 匹配页面初始化
        defaultDataCallBack.upPageRegular(this.getRegularList(), WormPageRegular.REGULAR_PAGE);
        this.setRegularList(defaultDataCallBack.findSeedListByType(WormPageRegular.REGULAR_PAGE));
    }

    public void execute() {

    }

    /**
     * 获取带抓取列表
     * @return
     */
    public List<WormPendPage> getPendListUrlPage() {
        List<WormPendPage> pendPageList = defaultDataCallBack.findPendPageList(DEFAULT_GET_PENDPAGE_NUM);
        if(pendPageList == null || pendPageList.size() == 0){
            return Collections.EMPTY_LIST;
        }
        for(int i = 0;i < pendPageList.size();i++){
            pendPageList.get(i).setQueueTime(new Date());
            pendPageList.get(i).setStatus(WormPendPage.STATUS_RUNING);
            pendPageList.get(i).setModifyTime(new Date());
        }
        defaultDataCallBack.upPendPageBatchById(pendPageList);
        return pendPageList;
    }


    /**
     * 新增已抓取成功页面
     * @param url
     */
    public void addGrabedPageUrl(String url) {
        if(StringUtils.isEmpty(url)){
            return;
        }
        WormGrabedPage pageData  = defaultDataCallBack.selectOneByUrl(url);
        if(pageData != null){
            pageData.setWormEndName(IpUtils.getIp());
            pageData.setModifyTime(new Date());
            defaultDataCallBack.upGrabedPageById(pageData);
            return;
        }
        WormGrabedPage wormGrabedPage = new WormGrabedPage();
        wormGrabedPage.setStatus(WormGrabedPage.STATUS_WAIT);
        wormGrabedPage.setPageUrl(url);
        wormGrabedPage.setCreateTime(new Date());
        wormGrabedPage.setModifyTime(new Date());
        wormGrabedPage.setWormEndName(IpUtils.getIp());
        wormGrabedPage.setModifyTime(new Date());
        wormGrabedPage.setEnabled(false);
        defaultDataCallBack.insertGrabedPage(wormGrabedPage);
    }

    /**
     * 获取新增带抓取链接
     * @param newGrabList
     */
    public void addGrabList(List<String> newGrabList, int maxNum) {
        if(newGrabList == null || newGrabList.size() == 0){
            return;
        }
        synchronized ("newsCachePendPage"){
            newsCachePendPageList.addAll(newGrabList);
            // 去重
            final List<String> handleNewCachePendPageList = new ArrayList<String>();
            for(String s: newsCachePendPageList){
                if(Collections.frequency(handleNewCachePendPageList, s) < 1) handleNewCachePendPageList.add(s);
            }
            newsCachePendPageList = handleNewCachePendPageList;
        }
    }

    /**
     * 独立线程新增带抓取任务
     *
     */
    public void newPendPageRuning(){
        Thread thread = new Thread(){
            public void run(){
                while(true){
                    List<String> queueList = new ArrayList<String>();
                    synchronized ("newsCachePendPage"){
                        if(newsCachePendPageList.size() == 0){
                            continue;
                        }
                        int toIndex = 100;
                        if(newsCachePendPageList.size() < 100){
                            toIndex = newsCachePendPageList.size();
                        }
                        if(toIndex != 0){
                            List<String> subList = newsCachePendPageList.subList(0, toIndex);
                            for(int i = 0;i<subList.size(); i++){
                                String pendPage = new String(subList.get(i));
                                queueList.add(pendPage);
                            }
                            newsCachePendPageList = newsCachePendPageList.subList(toIndex, newsCachePendPageList.size());
                        }
                    }
                    try{
                        if(queueList.size() != 0){
                            System.out.println("开始加入待抓取同步数据数>>" + queueList.size());
                            List<WormPendPage> pendPageList = new ArrayList<WormPendPage>();
                            for(String url:queueList){
                                WormPendPage pendPage = new WormPendPage();
                                pendPage.setCreateTime(new Date());
                                pendPage.setModifyTime(new Date());
                                pendPage.setEnabled(false);
                                pendPage.setWormName(getCrawlerName());
                                pendPage.setPageUrl(url);
                            }
                            defaultDataCallBack.upPendPageBatchById(pendPageList);
                            System.out.println("待抓取同步数据完成数>>" + queueList.size());
                        }
                    }catch (Exception e){
                        e.printStackTrace();
                    }
                }
            }
        };
        thread.start();
    }

    /**
     * 获取待抓取链接
     * @return
     */
    public synchronized WormPendPage getThreadGrabUrl(){
        if(getGrabUrlList() == null || getGrabUrlList().size() == 0){
            int count = defaultDataCallBack.selectPendPageCountByStatus(WormPendPage.STATUS_NORMAL);
            if(count == 0){
                return null;
            }
            DistributedLock.getLock("pendPageUrl_lock");
            List<WormPendPage> wormPendPageList = getPendListUrlPage();
            DistributedLock.alsoLock("pendPageUrl_lock");
            setGrabUrlList(wormPendPageList);
            return null;
        }
        WormPendPage grabUrl = getGrabUrlList().get(0);
        getGrabUrlList().remove(0);
        return grabUrl;
    }

    /**
     * 从源头开始抓取
     * @return
     */
    public void getHomePageInit(){
        DistributedLock.getLock("grabedHome_lock");
        int times = 60 * 24;
        Date nowDate = new Date();
        List<WormHomePage> homePageList = defaultDataCallBack.findHomePageList(new WormHomePage());
        if(homePageList != null && homePageList.size() != 0){
            // 查找指定时间还未重新更新的源头网站
            for(int i = 0;i<homePageList.size();i++){
                WormHomePage homePage = homePageList.get(i);
                if(homePage.getModifyTime() == null){
                    WormGrabedPage grabedPage = new WormGrabedPage();

                }
            }
        }
        DistributedLock.alsoLock("grabedHome_lock");
    }

    public void setDataObjMap(HashMap<String, Object> dataObjMap) {
        this.dataObjMap = dataObjMap;
    }
}
